<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2016-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Developing Pipelines</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="CDAP Pipelines" href="index.html" />
    <link rel="next" title="Developing Plugins" href="developing-plugins/index.html" />
    <link rel="prev" title="Post-run Plugins" href="plugins/post-run-plugins/index.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: developer-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../getting-started/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/sandbox/index.html">CDAP Sandbox</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html">二进制 Zip 文件</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html#cdap-sandbox">启动和停止 CDAP Sandbox</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/virtual-machine.html">虚拟机镜像</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/docker.html">Docker 镜像</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/quick-start.html">快速入门</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/dev-env.html">搭建开发环境</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/start-stop-cdap.html">启动和停止 CDAP</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/building-apps.html">构建并运行应用</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../overview/index.html"> 概述</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../overview/anatomy.html"> 大数据应用剖析</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/modes.html"> 模式和组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/abstractions.html"> 核心概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/interfaces.html"> 编程接口</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../building-blocks/index.html"> 抽象概念</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/core.html"> Core Abstractions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/applications.html"> Applications</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/datasets/index.html"> Datasets</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/overview.html"> Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/table.html"> Table API</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/fileset.html"> FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/partitioned-fileset.html"> Partitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/time-partitioned-fileset.html"> TimePartitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/system-custom.html"> System and Custom Datasets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/permissions.html"> Dataset Permissions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/cube.html"> Cube Dataset</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/mapreduce-programs.html"> MapReduce Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/plugins.html"> Plugins</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/schedules.html"> Schedules</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/secure-keys.html"> Secure Keys</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/services.html"> Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/spark-programs.html"> Spark Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workers.html"> Workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workflows.html"> Workflows</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/artifacts.html"> Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/program-lifecycle.html"> Program Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/namespaces.html"> Namespaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transaction-system.html"> Transaction System</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transactional-messaging-system.html"> Transactional Messaging System</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../metadata/index.html"> 元数据</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../metadata/system-metadata.html"> System Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/discovery-lineage.html"> Discovery and Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/field-lineage.html"> Field Level Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/audit-logging.html"> Audit Logging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/metadata-ui.html"> CDAP Metadata UI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/programmatic-metadata.html"> Accessing metadata programmatically</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> 数据流管道</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="concepts-design.html"> Concepts and Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="getting-started.html"> Getting Started</a></li>
<li class="toctree-l2"><a class="reference internal" href="studio.html"> CDAP Studio</a></li>
<li class="toctree-l2"><a class="reference internal" href="creating-pipelines.html"> Creating Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="running-pipelines.html"> Running Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugin-management.html"> Plugin Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugins/index.html"> Plugin Reference</a><ul>
<li class="toctree-l3"><a class="reference internal" href="plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
<li class="toctree-l2 current"><a class="current reference internal" href="#"> Developing Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="developing-plugins/index.html"> Developing Plugins</a><ul>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/plugin-basics.html">Plugin Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/creating-a-plugin.html">Creating a Plugin</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/presentation-plugins.html">Plugin Presentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/testing-plugins.html">Testing Plugins</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/packaging-plugins.html">Packaging Plugins</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="how-cdap-pipelines-work.html"> How CDAP Pipelines Work</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../cloud-runtimes/index.html"> 云平台运行</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/concepts/index.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/provisioners/index.html"> Provisioners</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/gcp-dataproc.html">Google Dataproc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/aws-emr.html">Amazon Elastic MapReduce</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/remote-hadoop.html">Remote Hadoop</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/profiles/index.html"> Profiles</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/creating-profiles.html">Creating Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/assigning-profiles.html">Assigning Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/admin-controls.html">Admin Controls</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/example/index.html"> Example</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../security/index.html"> 安全</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../security/client-authentication.html">Client Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-java.html">CDAP Authentication Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-python.html">CDAP Authentication Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/custom-authentication.html">Custom Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/authorization-extensions.html">Authorization Extensions</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../testing/index.html"> 测试和调试</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../testing/testing.html"> Testing a CDAP Application</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/debugging.html"> Debugging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/troubleshooting.html"> Troubleshooting</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../ingesting-tools/index.html"> 数据融合</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-java.html">CDAP Stream Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-python.html">CDAP Stream Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-ruby.html">CDAP Stream Client for Ruby</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-flume.html">CDAP Flume</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../data-exploration/index.html"> 数据探索</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/filesets.html"> Fileset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/tables.html"> Table Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/object-mapped-tables.html"> ObjectMappedTable Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/custom-datasets.html"> Custom Dataset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/hive-execution-engines.html"> Hive Execution Engines</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/index.html"> 高级主题</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../advanced/application-logback.html"> Application Logback</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/best-practices.html"> Best Practices</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/class-loading.html"> Class Loading</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/configuring-resources.html"> Configuring Program Resources</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/program-retry-policies.html"> Program Retry Policies</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="developing-pipelines">
<span id="cdap-pipelines-developing-pipelines"></span><h1>Developing Pipelines<a class="headerlink" href="#developing-pipelines" title="Permalink to this headline">🔗</a></h1>
<p>This section is intended for developers creating pipelines using command line tools. Users of
pipelines should refer to the documentation on <a class="reference internal" href="creating-pipelines.html#cdap-pipelines-creating-pipelines"><span class="std std-ref">creating pipelines</span></a>.</p>
<p>As mentioned in <a class="reference internal" href="creating-pipelines.html#cdap-pipelines-creating-pipelines"><span class="std std-ref">creating pipelines</span></a>, there are two
different methods for creating pipeline applications:</p>
<ol class="arabic simple">
<li>Using the CDAP Studio</li>
<li>Using command line tools (such as <code class="docutils literal notranslate"><span class="pre">curl</span></code>, the CDAP CLI, or the CDAP UI)</li>
</ol>
<p>Using comamnd line tools, pipelines can be created using the <span class="xref std std-ref">Lifecycle RESTful API</span>, the <span class="xref std std-ref">CDAP CLI</span>, or the <span class="xref std std-ref">CDAP UI</span>.</p>
<p>In order to create a pipeline application, a pipeline configuration (either as a file or
in-memory) is required that specifies the configuration of plugins to be used along with
their properties. (In the CDAP Studio, the user interface prompts you for the required
information.)</p>
<p>This section describes how to create such a configuration, and various aspects to be
considered in the design of pipelines.</p>
<div class="section" id="pipeline-configuration-file-format">
<span id="cdap-pipelines-developing-pipelines-configuration-file-format"></span><h2>Pipeline Configuration File Format<a class="headerlink" href="#pipeline-configuration-file-format" title="Permalink to this headline">🔗</a></h2>
<p>The configuration file format is JSON, containing these objects at the top level:</p>
<table border="1" class="docutils">
<colgroup>
<col width="10%" />
<col width="10%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Object</th>
<th class="head">Type</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">name</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">String</span></code></td>
<td>Name of the pipeline</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">artifact</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">Object</span></code></td>
<td>Specifies the artifact used to create the pipeline</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">config</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">Object</span></code></td>
<td>Configuration of the pipeline</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">__ui__</span></code></td>
<td><code class="docutils literal notranslate"><span class="pre">Object</span></code></td>
<td>Specifies the UI display of the pipeline</td>
</tr>
</tbody>
</table>
<p>The first three are required, while the <code class="docutils literal notranslate"><span class="pre">__ui__</span></code> is optional. It is added by the CDAP Application
Studio to store the display of the pipeline in the UI. The format of the <code class="docutils literal notranslate"><span class="pre">__ui__</span></code> object
is not described here, as it is internal to the CDAP Studio UI and subject to change.</p>
<p>Example (in JSON format):</p>
<div class="highlight container">
<pre class="literal-block">
{
  &quot;name&quot;: &quot;MyPipeline&quot;,
  &quot;artifact&quot;: {
    &quot;name&quot;: &quot;cdap-data-pipeline&quot;,
    &quot;version&quot;: &quot;6.1.1&quot;,
    &quot;scope&quot;: &quot;system&quot;
  },
  &quot;config&quot;: {
    . . .
    &quot;connections&quot;: [ . . . ],
    &quot;engine&quot;: &quot;mapreduce&quot;,
    &quot;postActions&quot;: [ . . . ],
    &quot;stages&quot;: [ . . . ],
    &quot;schedule&quot;: &quot;0 * * * *&quot;,
  },
  &quot;__ui__&quot;: {
    . . .
  }
}
</pre>
</div>
<div class="section" id="artifact-format">
<h3>Artifact Format<a class="headerlink" href="#artifact-format" title="Permalink to this headline">🔗</a></h3>
<p>The format of an <code class="docutils literal notranslate"><span class="pre">artifact</span></code> object:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Key</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">name</span></code></td>
<td>Name of the artifact</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">version</span></code></td>
<td>Allowable version(s) of the artifact</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">scope</span></code></td>
<td>Scope of the artifact, one of either <code class="docutils literal notranslate"><span class="pre">user</span></code> or <code class="docutils literal notranslate"><span class="pre">system</span></code></td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="configuration-format">
<h3>Configuration Format<a class="headerlink" href="#configuration-format" title="Permalink to this headline">🔗</a></h3>
<p>The format of the <code class="docutils literal notranslate"><span class="pre">config</span></code> object:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Key</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">connections</span></code></td>
<td>List of <code class="docutils literal notranslate"><span class="pre">connection</span></code> objects</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">engine</span></code></td>
<td>One of “mapreduce” or “spark”, used only in batch pipelines</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">postActions</span></code></td>
<td>List of <code class="docutils literal notranslate"><span class="pre">postaction</span></code> (post-run action) objects; optional, can be an empty list</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">instances</span></code></td>
<td>Positive integer of the number of worker instances, defaults to 1 if not specified,
used only in real-time pipelines</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">resources</span></code></td>
<td>A map defining the resources to be used for worker instance of pipeline, such as
<code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">&quot;memoryMB&quot;:</span> <span class="pre">2048,</span> <span class="pre">&quot;virtualCores&quot;:</span> <span class="pre">1</span> <span class="pre">}</span></code></td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">driverResources</span></code></td>
<td>A map defining the resources to be used for a database driver, such as username or password</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">schedule</span></code></td>
<td>String in <code class="docutils literal notranslate"><span class="pre">cron</span></code> file format, used only in batch pipelines</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">stages</span></code></td>
<td>List of <code class="docutils literal notranslate"><span class="pre">stage</span></code> objects</td>
</tr>
</tbody>
</table>
<p>The format of the <code class="docutils literal notranslate"><span class="pre">post-action</span></code> object is identical to that of a <code class="docutils literal notranslate"><span class="pre">stage</span></code> object, as
they are both a JSON representation of a plugin. However, only plugins of type
<code class="docutils literal notranslate"><span class="pre">postaction</span></code> can be included in the list for <code class="docutils literal notranslate"><span class="pre">postActions</span></code>. Each <code class="docutils literal notranslate"><span class="pre">postaction</span></code> object
must have a unique name.</p>
<p>The format of a <code class="docutils literal notranslate"><span class="pre">connection</span></code> object:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Key</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">from</span></code></td>
<td>String name of <code class="docutils literal notranslate"><span class="pre">stage</span></code> connected from</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">to</span></code></td>
<td>String name of <code class="docutils literal notranslate"><span class="pre">stage</span></code> connected to</td>
</tr>
</tbody>
</table>
<p>The format of <code class="docutils literal notranslate"><span class="pre">stage</span></code> and <code class="docutils literal notranslate"><span class="pre">postaction</span></code> objects:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Key</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">name</span></code></td>
<td>String name; these must be unique to distinguish stages in the <code class="docutils literal notranslate"><span class="pre">connections</span></code> object and
post-actions in the <code class="docutils literal notranslate"><span class="pre">postActions</span></code> list</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">plugin</span></code></td>
<td>Plugin object</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">errorDatasetName</span></code></td>
<td>Name of a dataset that any error messages will be written to; used by
<a class="reference internal" href="running-pipelines.html#cdap-pipelines-running-pipelines-error-record-handling"><span class="std std-ref">validating transform</span></a>
stages</td>
</tr>
</tbody>
</table>
<p>The format of a <code class="docutils literal notranslate"><span class="pre">plugin</span></code> object:</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Key</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">name</span></code></td>
<td>String name identifying the plugin</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">type</span></code></td>
<td>String type of plugin</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">artifact</span></code></td>
<td>Optional artifact object; if not included or defined, the highest version available is used</td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">properties</span></code></td>
<td>Map of properties, contents of which are determined by the particular plugin used</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="section" id="creating-a-batch-pipeline">
<span id="cdap-pipelines-developing-pipelines-creating-batch"></span><h2>Creating a Batch Pipeline<a class="headerlink" href="#creating-a-batch-pipeline" title="Permalink to this headline">🔗</a></h2>
<p>With a CDAP batch pipeline, a <code class="docutils literal notranslate"><span class="pre">schedule</span></code> property is required with a <code class="docutils literal notranslate"><span class="pre">cron</span></code> entry
specifying the frequency of the batch job run, such as every day or every hour.</p>
<p>For example, this JSON (when in a file such as <code class="docutils literal notranslate"><span class="pre">config.json</span></code>) provides a
configuration for a batch pipeline that runs every minute, reading data from a stream
<em>myStream</em> and writing to a dataset (<code class="docutils literal notranslate"><span class="pre">Table</span></code>) called <em>myTable</em>, without any transformations;
when the run completes, a post-run action send an email indicating that the run has completed:</p>
<div class="highlight container">
<pre class="literal-block">
{
  &quot;name&quot;: &quot;streamETLApp&quot;,
  &quot;artifact&quot;: {
    &quot;name&quot;: &quot;cdap-data-pipeline&quot;,
    &quot;version&quot;: &quot;6.1.1&quot;,
    &quot;scope&quot;: &quot;system&quot;
  },
  &quot;config&quot;: {
    &quot;schedule&quot;: &quot;* * * * *&quot;,
    &quot;engine&quot;: &quot;mapreduce&quot;,
    &quot;postActions&quot;: [
      {
        &quot;name&quot;: &quot;Email&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Email&quot;,
          &quot;type&quot;: &quot;postaction&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id1"><span class="problematic" id="id2">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;runCondition&quot;: &quot;completion&quot;,
            &quot;includeWorkflowToken&quot;: &quot;false&quot;,
            &quot;recipients&quot;: &quot;users&#64;example.com&quot;,
            &quot;sender&quot;: &quot;admin&#64;example.com&quot;,
            &quot;subject&quot;: &quot;Post-action Status&quot;,
            &quot;message&quot;: &quot;Completed run.&quot;
          }
        }
      }
    ],
    &quot;stages&quot;: [
      {
        &quot;name&quot;: &quot;streamSource&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Stream&quot;,
          &quot;type&quot;: &quot;batchsource&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id3"><span class="problematic" id="id4">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;system&quot;
          },
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;myStream&quot;,
            &quot;duration&quot;: &quot;1m&quot;,
            &quot;format&quot;: &quot;text&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;body\&quot;,\&quot;type\&quot;:\&quot;string\&quot;}]}&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;tableSink&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Table&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id5"><span class="problematic" id="id6">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;system&quot;
          },
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;myTable&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;ts\&quot;,\&quot;type\&quot;:\&quot;long\&quot;},{\&quot;name\&quot;:\&quot;body\&quot;,\&quot;type\&quot;:\&quot;string\&quot;}]}&quot;,
            &quot;schema.row.field&quot;: &quot;ts&quot;
          }
        }
      }
    ],
    &quot;connections&quot;: [
      {
        &quot;from&quot;: &quot;streamSource&quot;,
        &quot;to&quot;: &quot;tableSink&quot;
      }
    ]
  }
}
</pre>
</div>
<p>This pipeline launches a MapReduce program that runs every minute, reads data from the
<code class="docutils literal notranslate"><span class="pre">Stream</span></code> <em>myStream</em> and writes to a <code class="docutils literal notranslate"><span class="pre">Table</span></code> <em>myTable</em>. A <em>Table Sink</em> needs a row key field to
be specified and can use the timestamp of a stream event for that.</p>
<p>A pipeline configuration (the <code class="docutils literal notranslate"><span class="pre">config</span></code> object) consists of stages, connections and other
properties. The stages consist of a single source, zero (or more) transforms, and one (or
more) sink(s). Each of these stages is identified by a unique name and the plugin to be used.</p>
<p>A <code class="docutils literal notranslate"><span class="pre">plugin</span></code> object is specified by a plugin-name, type, a properties map and can
(optionally) specify the artifact. If the artifact is not specified, the pipeline will
choose the artifact with the highest version.</p>
<p>The connections field in the configuration defines the connections between the stages,
using the unique names of the stages. The pipeline defined by these connections must be a
directed acyclic graph (or DAG).</p>
<p>To create this pipeline, called <em>streamETLApp</em>, you can use either an HTTP RESTful API or
the CDAP CLI.</p>
<ul>
<li><p class="first">Using the <span class="xref std std-ref">Lifecycle RESTful API</span>:</p>

<script type="text/javascript">

  $(function tabbedparsedliteral58() {
    var tabs = ['linux', 'windows'];
    var mapping = {'windows': 'windows', 'linux': 'linux'};
    var tabSetID = 'linux-windows';
    for (var i = 0; i < tabs.length; i++) {
      var tab = tabs[i];
      $("#tabbedparsedliteral58 .example-tab-" + tab).click(changeExampleTab(tab, mapping, "tabbedparsedliteral58", tabSetID));
    }
  });

</script>
<div id="tabbedparsedliteral58" class="tabbed-parsed-literal dependent-linux-windows">
<ul class="tabbed-parsed-literal nav-tabs">
<li class="example-tab example-tab-linux active"><a href="#">Linux</a></li>
<li class="example-tab example-tab-windows "><a href="#">Windows</a></li>
</ul>

<div class="tab-contents">

<div class="tab-pane tab-pane-linux active">
<div class="code code-tab">
<div class="highlight-console">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">$</span> curl -w<span class="s2">&quot;\n&quot;</span> -X PUT localhost:11015/v3/namespaces/default/apps/streamETLApp <span class="se">\</span>
    -H <span class="s2">&quot;Content-Type: application/json&quot;</span> -d @config.json</span>

<span class="go">Deploy Complete</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
<div class="tab-pane tab-pane-windows ">
<div class="code code-tab">
<div class="highlight-shell-session">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">&gt;</span> curl -X PUT localhost:11015/v3/namespaces/default/apps/streamETLApp ^
<span class="go">    -H &quot;Content-Type: application/json&quot; -d @config.json</span>

<span class="go">Deploy Complete</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
</div>
</div>
</li>
<li><p class="first">Using the <span class="xref std std-ref">CDAP CLI</span>:</p>

<script type="text/javascript">

  $(function tabbedparsedliteral59() {
    var tabs = ['linux', 'windows'];
    var mapping = {'windows': 'windows', 'linux': 'linux'};
    var tabSetID = 'linux-windows';
    for (var i = 0; i < tabs.length; i++) {
      var tab = tabs[i];
      $("#tabbedparsedliteral59 .example-tab-" + tab).click(changeExampleTab(tab, mapping, "tabbedparsedliteral59", tabSetID));
    }
  });

</script>
<div id="tabbedparsedliteral59" class="tabbed-parsed-literal dependent-linux-windows">
<ul class="tabbed-parsed-literal nav-tabs">
<li class="example-tab example-tab-linux active"><a href="#">Linux</a></li>
<li class="example-tab example-tab-windows "><a href="#">Windows</a></li>
</ul>

<div class="tab-contents">

<div class="tab-pane tab-pane-linux active">
<div class="code code-tab">
<div class="highlight-console">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">$</span> cdap cli create app streamETLApp cdap-data-pipeline <span class="m">6</span>.1.1 system &lt;path-to-config.json&gt;

<span class="go">Successfully created application</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
<div class="tab-pane tab-pane-windows ">
<div class="code code-tab">
<div class="highlight-shell-session">
<!-- tabbed-parsed-literal start -->
<div class="highlight"><pre><span></span><span class="gp">&gt;</span> cdap cli create app streamETLApp cdap-data-pipeline <span class="m">6</span>.1.1 system &lt;path-to-config.json&gt;

<span class="go">Successfully created application</span>
</pre></div>
<!-- tabbed-parsed-literal end --></div>
</div>
</div>
</div>
</div>
</li>
</ul>
<p>where, in both cases, <code class="docutils literal notranslate"><span class="pre">config.json</span></code> is the file that contains the pipeline configuration shown above.</p>
</div>
<div class="section" id="creating-a-real-time-pipeline">
<span id="cdap-pipelines-developing-pipelines-creating-real-time"></span><h2>Creating a Real-Time Pipeline<a class="headerlink" href="#creating-a-real-time-pipeline" title="Permalink to this headline">🔗</a></h2>
<p>With a CDAP real-time pipeline, a <code class="docutils literal notranslate"><span class="pre">batchInterval</span></code> property is required
specifying the frequency at which the sources will create new micro batches of data.
The <code class="docutils literal notranslate"><span class="pre">batchInterval</span></code> must be a number followed by a time unit, with ‘s’ for seconds,
‘m’ for minutes, and ‘h’ for hours. For example, a value of ’10s’ will be interpreted
as ten seconds. This means that every ten seconds, a new micro batch of data will be generated.</p>
<p>To create a real-time pipeline that reads from a source such as Twitter and writes to a
CDAP Table after performing a projection transformation, you can use a configuration such as:</p>
<div class="highlight container">
<pre class="literal-block">
{
  &quot;name&quot;: &quot;twitterToStream&quot;,
  &quot;artifact&quot;: {
    &quot;name&quot;: &quot;cdap-data-streams&quot;,
    &quot;version&quot;: &quot;6.1.1&quot;,
    &quot;scope&quot;: &quot;system&quot;
  },
  &quot;config&quot;: {
    &quot;batchInterval&quot;: &quot;10s&quot;,
    &quot;stages&quot;: [
      {
        &quot;name&quot;: &quot;twitterSource&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Twitter&quot;,
          &quot;type&quot;: &quot;streamingsource&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;spark-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id7"><span class="problematic" id="id8">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;system&quot;
          },
          &quot;properties&quot;: {
            &quot;referenceName&quot;: &quot;xxx&quot;,
            &quot;AccessToken&quot;: &quot;xxx&quot;,
            &quot;AccessTokenSecret&quot;: &quot;xxx&quot;,
            &quot;ConsumerKey&quot;: &quot;xxx&quot;,
            &quot;ConsumerSecret&quot;: &quot;xxx&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;dropProjector&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Projection&quot;,
          &quot;type&quot;: &quot;transform&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id9"><span class="problematic" id="id10">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;system&quot;
          },
          &quot;properties&quot;: {
            &quot;drop&quot;: &quot;lang,time,favCount,source,geoLat,geoLong,isRetweet&quot;,
            &quot;rename&quot;: &quot;message:tweet,rtCount:retCount&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;tableSink&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Table&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id11"><span class="problematic" id="id12">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;system&quot;
          },
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;myTable&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;id\&quot;,\&quot;type\&quot;:\&quot;long\&quot;},{\&quot;name\&quot;:\&quot;tweet\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},{\&quot;name\&quot;:\&quot;retCount\&quot;,\&quot;type\&quot;:\&quot;int\&quot;}]}&quot;,
            &quot;schema.row.field&quot;: &quot;id&quot;
          }
        }
      }
    ],
    &quot;connections&quot;: [
      {
        &quot;from&quot;: &quot;twitterSource&quot;,
        &quot;to&quot;: &quot;dropProjector&quot;
      },
      {
        &quot;from&quot;: &quot;dropProjector&quot;,
        &quot;to&quot;: &quot;tableSink&quot;
      }
    ]
  }
}
</pre>
</div>
<p>In the example code above, we will use a <em>ProjectionTransform</em> (a type of transform) to
drop and rename selected columns in the incoming data. A <em>TableSink</em> in the final step
requires a schema property that it will use to set the columns and row field for
writing the data to a CDAP Table.</p>
</div>
<div class="section" id="non-linear-executions-in-pipelines">
<h2>Non-linear Executions in Pipelines<a class="headerlink" href="#non-linear-executions-in-pipelines" title="Permalink to this headline">🔗</a></h2>
<p>CDAP pipelines supports directed acyclic graphs in pipelines, which allows for the non-linear
execution of pipeline stages.</p>
<div class="section" id="fork-in-pipeline">
<h3>Fork in Pipeline<a class="headerlink" href="#fork-in-pipeline" title="Permalink to this headline">🔗</a></h3>
<p>In this example, a pipeline reads from the stream <code class="docutils literal notranslate"><span class="pre">purchaseStats</span></code>. It writes the stream
events to the table <code class="docutils literal notranslate"><span class="pre">replicaTable</span></code>, while at the same time it writes just the
<code class="docutils literal notranslate"><span class="pre">userIds</span></code> to the <code class="docutils literal notranslate"><span class="pre">usersTable</span></code> when a user’s purchase price is greater than 1000. This
filtering logic is applied by using an included script in the step
<code class="docutils literal notranslate"><span class="pre">spendingUsersScript</span></code>:</p>
<a class="reference internal image-reference" href="../_images/fork-in-pipeline.png"><img alt="../_images/fork-in-pipeline.png" class="align-center" src="../_images/fork-in-pipeline.png" style="width: 6in;" /></a>
<div class="highlight container">
<pre class="literal-block">
{
  &quot;name&quot;: &quot;forkedPipeline&quot;,
  &quot;artifact&quot;: {
    &quot;name&quot;: &quot;cdap-data-pipeline&quot;,
    &quot;version&quot;: &quot;6.1.1&quot;,
    &quot;scope&quot;: &quot;SYSTEM&quot;
  },
  &quot;config&quot;: {
    &quot;schedule&quot;: &quot;* * * * *&quot;,
    &quot;engine&quot;: &quot;mapreduce&quot;,
    &quot;postActions&quot;: [],
    &quot;stages&quot;: [
      {
        &quot;name&quot;: &quot;purchaseStats&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Stream&quot;,
          &quot;type&quot;: &quot;batchsource&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id13"><span class="problematic" id="id14">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;testStream&quot;,
            &quot;duration&quot;: &quot;1d&quot;,
            &quot;format&quot;: &quot;csv&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;userId\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},{\&quot;name\&quot;:\&quot;purchaseItem\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},{\&quot;name\&quot;:\&quot;purchasePrice\&quot;,\&quot;type\&quot;:\&quot;long\&quot;}]}&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;replicaTable&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Table&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id15"><span class="problematic" id="id16">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;replicaTable&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;userId\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},{\&quot;name\&quot;:\&quot;purchaseItem\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},{\&quot;name\&quot;:\&quot;purchasePrice\&quot;,\&quot;type\&quot;:\&quot;long\&quot;}]}&quot;,
            &quot;schema.row.field&quot;: &quot;userId&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;usersTable&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Table&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id17"><span class="problematic" id="id18">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;targetCustomers&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;userId\&quot;,\&quot;type\&quot;:\&quot;string\&quot;}]}&quot;,
            &quot;schema.row.field&quot;: &quot;userId&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;spendingUsersScript&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;JavaScript&quot;,
          &quot;type&quot;: &quot;transform&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id19"><span class="problematic" id="id20">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;script&quot;: &quot;function transform(input, emitter, context) {\n  if (input.purchasePrice &gt; 1000) {\n  emitter.emit(input);\n  }  \n}&quot;
          }
        }
      }
    ],
   &quot;connections&quot;: [
      {
        &quot;from&quot;: &quot;purchaseStats&quot;,
        &quot;to&quot;: &quot;replicaTable&quot;
      },
      {
        &quot;from&quot;: &quot;purchaseStats&quot;,
        &quot;to&quot;: &quot;spendingUsersScript&quot;
      },
      {
        &quot;from&quot;: &quot;spendingUsersScript&quot;,
        &quot;to&quot;: &quot;usersTable&quot;
      }
    ]
  }
}
</pre>
</div>
<p>Pipeline connections can be configured to fork from a stage, with the output of the stage
sent to two or more configured stages; in the above example, the output record from
<code class="docutils literal notranslate"><span class="pre">purchaseStats</span></code> will be sent to both <code class="docutils literal notranslate"><span class="pre">replicaTable</span></code> and <code class="docutils literal notranslate"><span class="pre">spendingUsersScript</span></code> stages.</p>
</div>
<div class="section" id="merging-stages-in-pipeline">
<h3>Merging Stages in Pipeline<a class="headerlink" href="#merging-stages-in-pipeline" title="Permalink to this headline">🔗</a></h3>
<p>Forked transform stages can merge together at a transform or a sink stage.</p>
<p>A merge does not join, or modify records in any way. It simply means that multiple stages
can write to the same stage. The only requirement is that all stages must output records
of the same schema to the merging stage. Note that the order of records sent from the
forked stages to the merging stage will not be defined.</p>
<p>In this next example, <code class="docutils literal notranslate"><span class="pre">purchaseStream</span></code> has purchase data with fields <code class="docutils literal notranslate"><span class="pre">userid</span></code>,
<code class="docutils literal notranslate"><span class="pre">item</span></code>, <code class="docutils literal notranslate"><span class="pre">count</span></code>, and <code class="docutils literal notranslate"><span class="pre">price</span></code>. The stream events source stage <code class="docutils literal notranslate"><span class="pre">purchaseStream</span></code>
forks, and records are sent to both of the transforms <code class="docutils literal notranslate"><span class="pre">userRewards</span></code> and <code class="docutils literal notranslate"><span class="pre">itemRewards</span></code>.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">userRewards</span></code> transform script looks up valued customers in the table
<code class="docutils literal notranslate"><span class="pre">hvCustomers</span></code>, to check if <code class="docutils literal notranslate"><span class="pre">userid</span></code> is a valued customer and assigns higher rewards if
they are. After calculating the rewards, this transform sends an output record in the
format <code class="docutils literal notranslate"><span class="pre">userid(string),</span> <span class="pre">rewards(double)</span></code>.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">itemRewards</span></code> transform script awards higher rewards for bulk purchases and sends
output records in the same format, <code class="docutils literal notranslate"><span class="pre">userid(string),</span> <span class="pre">rewards(double)</span></code>.</p>
<p>The rewards records are merged at the sink <code class="docutils literal notranslate"><span class="pre">rewardsSink</span></code>; note that the incoming schema
from the transforms <code class="docutils literal notranslate"><span class="pre">userRewards</span></code> and <code class="docutils literal notranslate"><span class="pre">itemRewards</span></code> are the same, and that the order
of received records will vary.</p>
<a class="reference internal image-reference" href="../_images/merge-in-pipeline.png"><img alt="../_images/merge-in-pipeline.png" class="align-center" src="../_images/merge-in-pipeline.png" style="width: 8in;" /></a>
<div class="highlight container">
<pre class="literal-block">
{
  &quot;name&quot;: &quot;mergedPipeline&quot;,
  &quot;artifact&quot;: {
      &quot;name&quot;: &quot;cdap-data-pipeline&quot;,
      &quot;version&quot;: &quot;6.1.1&quot;,
      &quot;scope&quot;: &quot;SYSTEM&quot;
  },
  &quot;config&quot;: {
    &quot;schedule&quot;: &quot;* * * * *&quot;,
    &quot;engine&quot;: &quot;mapreduce&quot;,
    &quot;postActions&quot;: [],
    &quot;stages&quot;: [
      {
        &quot;name&quot;: &quot;purchaseStream&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Stream&quot;,
          &quot;type&quot;: &quot;batchstream&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id21"><span class="problematic" id="id22">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;format&quot;: &quot;csv&quot;,
            &quot;schema&quot;: &quot;{
              &quot;type&quot;:&quot;record&quot;,
              &quot;name&quot;:&quot;etlSchemaBody&quot;,
              &quot;fields&quot;:[
                {&quot;name&quot;:&quot;userid&quot;,&quot;type&quot;:&quot;string&quot;},
                {&quot;name&quot;:&quot;item&quot;,&quot;type&quot;:&quot;string&quot;},
                {&quot;name&quot;:&quot;count&quot;,&quot;type&quot;:&quot;int&quot;},
                {&quot;name&quot;:&quot;price&quot;,&quot;type&quot;:&quot;long&quot;}
              ]
            }&quot;,
            &quot;name&quot;: &quot;purchases&quot;,
            &quot;duration&quot;: &quot;1d&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;userRewards&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Script&quot;,
          &quot;type&quot;: &quot;transform&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id23"><span class="problematic" id="id24">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;script&quot;: &quot;function transform(input, context) {
              var rewards = 5;
              if (context.getLookup('hvCustomers').lookup(input.userid) !== null) {
                context.getLogger().info(&quot;user &quot; + input.userid + &quot; is a valued customer&quot;);
                rewards = 100;
              } else {
                context.getLogger().info(&quot;user &quot; + input.userid + &quot; is not a valued customer&quot;);
              }
              return {'userid': input.userid, 'rewards': rewards};
            }&quot;,
            &quot;schema&quot;: &quot;{
              &quot;type&quot;:&quot;record&quot;,
              &quot;name&quot;:&quot;etlSchemaBody&quot;,
              &quot;fields&quot;:[
                {&quot;name&quot;:&quot;userid&quot;,&quot;type&quot;:&quot;string&quot;},
                {&quot;name&quot;:&quot;rewards&quot;,&quot;type&quot;:&quot;double&quot;}
              ]
            }&quot;,
            &quot;lookup&quot;: &quot;{&quot;tables&quot;:{&quot;hvCustomers&quot;:{&quot;type&quot;:&quot;DATASET&quot;,&quot;datasetProperties&quot;:{}}}}&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;itemRewards&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Script&quot;,
          &quot;type&quot;: &quot;transform&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id25"><span class="problematic" id="id26">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;script&quot;: &quot;function transform(input, context) {
              var rewards = 5;
              if (input.count &gt; 20) {
                rewards = 50;
              }
              return {'userid':input.userid, 'rewards':rewards};
            }&quot;,
            &quot;schema&quot;: &quot;{
              &quot;type&quot;:&quot;record&quot;,
              &quot;name&quot;:&quot;etlSchemaBody&quot;,
              &quot;fields&quot;:[
                {&quot;name&quot;:&quot;userid&quot;,&quot;type&quot;:&quot;string&quot;},
                {&quot;name&quot;:&quot;rewards&quot;,&quot;type&quot;:&quot;double&quot;}
              ]
            }&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;rewardsSink&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;TPFSAvro&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id27"><span class="problematic" id="id28">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;schema&quot;: &quot;{
              &quot;type&quot;:&quot;record&quot;,
              &quot;name&quot;:&quot;etlSchemaBody&quot;,
              &quot;fields&quot;:[
                {&quot;name&quot;:&quot;userid&quot;,&quot;type&quot;:&quot;string&quot;},
                {&quot;name&quot;:&quot;rewards&quot;,&quot;type&quot;:&quot;double&quot;}
              ]
            }&quot;
          }
        }
      }
    ],
    &quot;connections&quot;: [
      {
        &quot;from&quot;: &quot;purchaseStream&quot;,
        &quot;to&quot;: &quot;userRewards&quot;
      },
      {
        &quot;from&quot;: &quot;userRewards&quot;,
        &quot;to&quot;: &quot;rewardsSink&quot;
      },
      {
        &quot;from&quot;: &quot;purchaseStream&quot;,
        &quot;to&quot;: &quot;itemRewards&quot;
      },
      {
        &quot;from&quot;: &quot;itemRewards&quot;,
        &quot;to&quot;: &quot;rewardsSink&quot;
      }
    ],
  }
}
</pre>
</div>
</div>
</div>
<div class="section" id="sample-pipeline-configurations">
<h2>Sample Pipeline Configurations<a class="headerlink" href="#sample-pipeline-configurations" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="database-source-and-sink">
<h3>Database Source and Sink<a class="headerlink" href="#database-source-and-sink" title="Permalink to this headline">🔗</a></h3>
<p>Sample configuration for using a <em>Database Source</em> and a <em>Database Sink</em>:</p>
<div class="highlight container">
<pre class="literal-block">
{
  &quot;artifact&quot;: {
    &quot;name&quot;: &quot;cdap-data-pipeline&quot;,
    &quot;version&quot;: &quot;6.1.1&quot;,
    &quot;scope&quot;: &quot;system&quot;
  },
  &quot;config&quot;: {
    &quot;schedule&quot;: &quot;* * * * *&quot;,
    &quot;stages&quot;: [
      {
        &quot;name&quot;: &quot;databaseSource&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Database&quot;,
          &quot;type&quot;: &quot;batchsource&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id29"><span class="problematic" id="id30">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;importQuery&quot;: &quot;select id,name,age from my_table&quot;,
            &quot;countQuery&quot;: &quot;select count(id) from my_table&quot;,
            &quot;connectionString&quot;: &quot;jdbc:mysql://localhost:3306/test&quot;,
            &quot;tableName&quot;: &quot;src_table&quot;,
            &quot;user&quot;: &quot;my_user&quot;,
            &quot;password&quot;: &quot;my_password&quot;,
            &quot;jdbcPluginName&quot;: &quot;jdbc_plugin_name_defined_in_jdbc_plugin_json_config&quot;,
            &quot;jdbcPluginType&quot;: &quot;jdbc_plugin_type_defined_in_jdbc_plugin_json_config&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;databaseSink&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Database&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id31"><span class="problematic" id="id32">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;SYSTEM&quot;
          },
          &quot;properties&quot;: {
            &quot;columns&quot;: &quot;id,name,age&quot;,
            &quot;connectionString&quot;: &quot;jdbc:mysql://localhost:3306/test&quot;,
            &quot;tableName&quot;: &quot;dest_table&quot;,
            &quot;user&quot;: &quot;my_user&quot;,
            &quot;password&quot;: &quot;my_password&quot;,
            &quot;jdbcPluginName&quot;: &quot;jdbc_plugin_name_defined_in_jdbc_plugin_json_config&quot;,
            &quot;jdbcPluginType&quot;: &quot;jdbc_plugin_type_defined_in_jdbc_plugin_json_config&quot;
          }
        }
      }
    ],
    &quot;connections&quot;: [
      {
        &quot;from&quot;: &quot;databaseSource&quot;,
        &quot;to&quot;: &quot;databaseSink&quot;
      }
    ]
  }
}
</pre>
</div>
</div>
<div class="section" id="kafka-source">
<h3>Kafka Source<a class="headerlink" href="#kafka-source" title="Permalink to this headline">🔗</a></h3>
<p>A Kafka cluster needs to be available, and certain minimum properties specified when
creating the source:</p>
<div class="highlight container">
<pre class="literal-block">
{
  &quot;name&quot;: &quot;KafkaPipeline&quot;,
  &quot;artifact&quot;: {
    &quot;name&quot;: &quot;cdap-data-streams&quot;,
    &quot;version&quot;: &quot;6.1.1&quot;,
    &quot;scope&quot;: &quot;system&quot;
  },
  &quot;config&quot;: {
    &quot;batchInterval&quot;: &quot;1s&quot;,
    &quot;connections&quot;: [
      {
        &quot;from&quot;: &quot;kafkaSource&quot;,
        &quot;to&quot;: &quot;tableSink&quot;
      }
    ],
    &quot;stages&quot;: [
      {
        &quot;name&quot;: &quot;kafkaSource&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Kafka&quot;,
          &quot;type&quot;: &quot;streamingsource&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;kafka-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id33"><span class="problematic" id="id34">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;USER&quot;
          },
          &quot;properties&quot;: {
            &quot;referenceName&quot;: &quot;purchasesTopic&quot;,
            &quot;brokers&quot;: &quot;localhost:9092&quot;,
            &quot;topics&quot;: &quot;purchases&quot;,
            &quot;format&quot;: &quot;csv&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;id\&quot;,\&quot;type\&quot;:\&quot;long\&quot;},{\&quot;name\&quot;:\&quot;customer\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},{\&quot;name\&quot;:\&quot;item\&quot;,\&quot;type\&quot;:\&quot;string\&quot;}]}&quot;
          }
        }
      },
      {
        &quot;name&quot;: &quot;tableSink&quot;,
        &quot;plugin&quot;: {
          &quot;name&quot;: &quot;Table&quot;,
          &quot;type&quot;: &quot;batchsink&quot;,
          &quot;artifact&quot;: {
            &quot;name&quot;: &quot;core-plugins&quot;,
            &quot;version&quot;: &quot;<a href="#id35"><span class="problematic" id="id36">|cdap-pipelines-version|</span></a>&quot;,
            &quot;scope&quot;: &quot;system&quot;
          },
          &quot;properties&quot;: {
            &quot;name&quot;: &quot;myTable&quot;,
            &quot;schema&quot;: &quot;{\&quot;type\&quot;:\&quot;record\&quot;,\&quot;name\&quot;:\&quot;etlSchemaBody\&quot;,\&quot;fields\&quot;:[{\&quot;name\&quot;:\&quot;id\&quot;,\&quot;type\&quot;:\&quot;long\&quot;},{\&quot;name\&quot;:\&quot;customer\&quot;,\&quot;type\&quot;:\&quot;string\&quot;},{\&quot;name\&quot;:\&quot;item\&quot;,\&quot;type\&quot;:\&quot;string\&quot;}]}&quot;
            &quot;schema.row.field&quot;: &quot;id&quot;
          }
        }
      }
    ]
  }
}
</pre>
</div>
</div>
<div class="section" id="prebuilt-jars">
<h3>Prebuilt JARs<a class="headerlink" href="#prebuilt-jars" title="Permalink to this headline">🔗</a></h3>
<p>In a case where you’d like to use prebuilt third-party JARs (such as a JDBC driver) as a
plugin, please refer to the section on <a class="reference internal" href="plugin-management.html#cdap-pipelines-plugin-management-third-party-plugins"><span class="std std-ref">Deploying Third-Party Jars</span></a>.</p>
</div>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Post-run Plugins" href="plugins/post-run-plugins/index.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Developing Plugins" href="developing-plugins/index.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>